From 913b98fd6fe814ee4c719a90a0642497c9d3d0dc Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 15 Dec 2016 22:13:36 +0100 Subject: [PATCH 1/8] fix `make` concurrency build (#277) --- Makefile | 4 +++- NEWS | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 48e6752..c77f697 100644 --- a/Makefile +++ b/Makefile @@ -52,7 +52,9 @@ endif .PHONY: default all lib lz4 clean test versionsTest examples -default: lib lz4-release +default: + @$(MAKE) -C $(LZ4DIR) + @$(MAKE) -C $(PRGDIR) all: @$(MAKE) -C $(LZ4DIR) $@ diff --git a/NEWS b/NEWS index 292cd27..ff62ca8 100644 --- a/NEWS +++ b/NEWS @@ -2,6 +2,7 @@ v1.7.5 cli : fix minor notification when using -r recursive mode lz4cat : fix : works with relative path (#284) and stdin (#285) (reported by @beiDei8z) doc : markdown version of man page, by Takayuki Matsuoka (#279) +build : Makefile : fix concurrency lib+exe build (#277) v1.7.4.2 fix : Makefile : release build compatible with PIE and customized compilation directives provided through environment variables (#274, reported by Antoine Martin) From 385cb4f539134b3cb855a29eef99a4a876f728d2 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 21 Dec 2016 13:18:02 +0100 Subject: [PATCH 2/8] minor update Makefile --- Makefile | 1 + lib/Makefile | 26 +++++++++++++++----------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index c77f697..647ab4e 100644 --- a/Makefile +++ b/Makefile @@ -55,6 +55,7 @@ endif default: @$(MAKE) -C $(LZ4DIR) @$(MAKE) -C $(PRGDIR) + @cp $(PRGDIR)/lz4$(EXT) . all: @$(MAKE) -C $(LZ4DIR) $@ diff --git a/lib/Makefile b/lib/Makefile index 2b47fb0..8426330 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -71,6 +71,8 @@ else SHARED_EXT_VER = $(SHARED_EXT).$(LIBVER) endif +LIBLZ4 = liblz4.$(SHARED_EXT_VER) + default: lib-release lib-release: liblz4.a liblz4 @@ -90,18 +92,20 @@ ifeq ($(BUILD_STATIC),yes) @$(AR) rcs $@ *.o endif -liblz4: *.c +$(LIBLZ4): *.c @echo compiling dynamic library $(LIBVER) ifneq (,$(filter Windows%,$(OS))) @$(CC) $(FLAGS) -DLZ4_DLL_EXPORT=1 -shared $^ -o dll\$@.dll dlltool -D dll\liblz4.dll -d dll\liblz4.def -l dll\liblz4.lib else - @$(CC) $(FLAGS) -shared $^ -fPIC $(SONAME_FLAGS) -o $@.$(SHARED_EXT_VER) + @$(CC) $(FLAGS) -shared $^ -fPIC $(SONAME_FLAGS) -o $@ @echo creating versioned links - @ln -sf $@.$(SHARED_EXT_VER) $@.$(SHARED_EXT_MAJOR) - @ln -sf $@.$(SHARED_EXT_VER) $@.$(SHARED_EXT) + @ln -sf $@ liblz4.$(SHARED_EXT_MAJOR) + @ln -sf $@ liblz4.$(SHARED_EXT) endif +liblz4: $(LIBLZ4) + clean: @$(RM) core *.o *.a *.$(SHARED_EXT) liblz4.pc dll/liblz4.dll dll/liblz4.lib @echo Cleaning library completed @@ -123,21 +127,21 @@ liblz4.pc: liblz4.pc.in Makefile install: lib liblz4.pc @install -d -m 755 $(DESTDIR)$(LIBDIR)/pkgconfig/ $(DESTDIR)$(INCLUDEDIR)/ @install -m 755 liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR) - @ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_MAJOR) - @ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT) + @install -m 755 liblz4.$(SHARED_EXT_MAJOR) $(DESTDIR)$(LIBDIR) + @install -m 755 liblz4.$(SHARED_EXT) $(DESTDIR)$(LIBDIR) @install -m 644 liblz4.pc $(DESTDIR)$(LIBDIR)/pkgconfig/ ifeq ($(BUILD_STATIC),yes) - @install -m 644 liblz4.a $(DESTDIR)$(LIBDIR)/liblz4.a + @install -m 644 liblz4.a $(DESTDIR)$(LIBDIR) endif - @install -m 644 lz4.h $(DESTDIR)$(INCLUDEDIR)/lz4.h - @install -m 644 lz4hc.h $(DESTDIR)$(INCLUDEDIR)/lz4hc.h - @install -m 644 lz4frame.h $(DESTDIR)$(INCLUDEDIR)/lz4frame.h + @install -m 644 lz4.h $(DESTDIR)$(INCLUDEDIR) + @install -m 644 lz4hc.h $(DESTDIR)$(INCLUDEDIR) + @install -m 644 lz4frame.h $(DESTDIR)$(INCLUDEDIR) @echo lz4 static and shared libraries installed uninstall: + @$(RM) $(DESTDIR)$(LIBDIR)/pkgconfig/liblz4.pc @$(RM) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT) @$(RM) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_MAJOR) - @$(RM) $(DESTDIR)$(LIBDIR)/pkgconfig/liblz4.pc @$(RM) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_VER) @$(RM) $(DESTDIR)$(LIBDIR)/liblz4.a @$(RM) $(DESTDIR)$(INCLUDEDIR)/lz4.h From 7cf0bb97b2a988cb17435780d19e145147dd9f70 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 21 Dec 2016 14:18:01 +0100 Subject: [PATCH 3/8] LZ4F_compressBound(0) provides upper bound for LZ4F_flush() and LZ4F_compressEnd() [#290, suggested by @vtermanis] --- lib/lz4frame.c | 15 +++++++++++---- lib/lz4frame.h | 23 ++++++++++++----------- tests/frametest.c | 7 +++++++ 3 files changed, 30 insertions(+), 15 deletions(-) diff --git a/lib/lz4frame.c b/lib/lz4frame.c index 3c2b788..626ac98 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -158,7 +158,7 @@ static void LZ4F_writeLE64 (void* dst, U64 value64) #define LZ4F_BLOCKSIZEID_DEFAULT LZ4F_max64KB static const size_t minFHSize = 7; -static const size_t maxFHSize = 15; /* max Frame Header Size */ +static const size_t maxFHSize = LZ4F_HEADER_SIZE_MAX; /* 15 */ static const size_t BHSize = 4; @@ -254,20 +254,27 @@ static LZ4F_blockSizeID_t LZ4F_optimalBSID(const LZ4F_blockSizeID_t requestedBSI return requestedBSID; } +/* LZ4F_compressBound() : + * Provides dstCapacity given a srcSize to guarantee operation success in worst case situations. + * prefsPtr is optional : you can provide NULL as argument, preferences will be set to cover worst case scenario. + * Result is always the same for a srcSize and prefsPtr, so it can be trusted to size reusable buffers. + * When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations. + */ static size_t LZ4F_compressBound_internal(size_t srcSize, const LZ4F_preferences_t* preferencesPtr, size_t alreadyBuffered) { LZ4F_preferences_t prefsNull; memset(&prefsNull, 0, sizeof(prefsNull)); prefsNull.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; /* worst case */ { const LZ4F_preferences_t* const prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr; + U32 const flush = prefsPtr->autoFlush | (srcSize==0); LZ4F_blockSizeID_t const bid = prefsPtr->frameInfo.blockSizeID; size_t const blockSize = LZ4F_getBlockSize(bid); size_t const maxBuffered = blockSize - 1; size_t const bufferedSize = MIN(alreadyBuffered, maxBuffered); size_t const maxSrcSize = srcSize + bufferedSize; unsigned const nbFullBlocks = (unsigned)(maxSrcSize / blockSize); - size_t const partialBlockSize = srcSize & (blockSize-1); - size_t const lastBlockSize = prefsPtr->autoFlush ? partialBlockSize : 0; + size_t const partialBlockSize = (srcSize - (srcSize==0)) & (blockSize-1); /* 0 => -1 == MAX => blockSize-1 */ + size_t const lastBlockSize = flush ? partialBlockSize : 0; unsigned const nbBlocks = nbFullBlocks + (lastBlockSize>0); size_t const blockHeaderSize = 4; /* default, without block CRC option (which cannot be generated with current API) */ @@ -398,7 +405,7 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp /*! LZ4F_compressBegin() : * will write the frame header into dstBuffer. - * dstBuffer must be large enough to accommodate a header (dstCapacity). Maximum header size is LZ4F_MAXHEADERFRAME_SIZE bytes. + * dstBuffer must be large enough to accommodate a header (dstCapacity). Maximum header size is LZ4F_HEADER_SIZE_MAX bytes. * @return : number of bytes written into dstBuffer for the header * or an error code (can be tested using LZ4F_isError()) */ diff --git a/lib/lz4frame.h b/lib/lz4frame.h index 3104d2e..a4a4ce6 100644 --- a/lib/lz4frame.h +++ b/lib/lz4frame.h @@ -206,10 +206,11 @@ LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx); /* Compression */ +#define LZ4F_HEADER_SIZE_MAX 15 LZ4FLIB_API size_t LZ4F_compressBegin(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_preferences_t* prefsPtr); /* LZ4F_compressBegin() : * will write the frame header into dstBuffer. - * dstBuffer must be large enough to accommodate a header. Maximum header size is 15 bytes. + * dstCapacity must be large enough to store the header. Maximum header size is LZ4F_HEADER_SIZE_MAX bytes. * `prefsPtr` is optional : you can provide NULL as argument, all preferences will then be set to default. * @return : number of bytes written into dstBuffer for the header * or an error code (which can be tested using LZ4F_isError()) @@ -217,20 +218,20 @@ LZ4FLIB_API size_t LZ4F_compressBegin(LZ4F_cctx* cctx, void* dstBuffer, size_t d LZ4FLIB_API size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* prefsPtr); /* LZ4F_compressBound() : - * Provides the minimum size of Dst buffer given srcSize to handle worst case situations. - * Different preferences can produce different results. - * prefsPtr is optional : you can provide NULL as argument, all preferences will then be set to cover worst case. - * This function includes frame termination cost (4 bytes, or 8 if frame checksum is enabled) + * Provides dstCapacity given a srcSize to guarantee operation success in worst case situations. + * prefsPtr is optional : you can provide NULL as argument, preferences will be set to cover worst case scenario. + * Result is always the same for a srcSize and prefsPtr, so it can be trusted to size reusable buffers. + * When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations. */ LZ4FLIB_API size_t LZ4F_compressUpdate(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* cOptPtr); /* LZ4F_compressUpdate() : * LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary. - * An important rule is that dstBuffer MUST be large enough (dstCapacity) to ensure compression completion even in worst case. - * This value is provided by using LZ4F_compressBound(). + * An important rule is that dstCapacity MUST be large enough to ensure operation success even in worst case situations. + * This value is provided by LZ4F_compressBound(). * If this condition is not respected, LZ4F_compress() will fail (result is an errorCode). * LZ4F_compressUpdate() doesn't guarantee error recovery. When an error occurs, compression context must be freed or resized. - * `cOptPtr` is optional : it's possible to provide NULL, all options will be set to default. + * `cOptPtr` is optional : NULL can be provided, in which case all options are set to default. * @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered). * or an error code if it fails (which can be tested using LZ4F_isError()) */ @@ -245,12 +246,12 @@ LZ4FLIB_API size_t LZ4F_flush(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapaci * or an error code if it fails (which can be tested using LZ4F_isError()) */ -LZ4FLIB_API size_t LZ4F_compressEnd(LZ4F_cctx* cctx, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* cOptPtr); +LZ4FLIB_API size_t LZ4F_compressEnd(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr); /* LZ4F_compressEnd() : - * To properly finish the compressed frame, invoke LZ4F_compressEnd(). + * To properly finish an LZ4 frame, invoke LZ4F_compressEnd(). * It will flush whatever data remained within `cctx` (like LZ4_flush()) * and properly finalize the frame, with an endMark and a checksum. - * `cOptPtr` is optional : it's possible to provide NULL, all options will be set to default. + * `cOptPtr` is optional : NULL can be provided, in which case all options will be set to default. * @return : number of bytes written into dstBuffer (necessarily >= 4 (endMark), or 8 if optional frame checksum is enabled) * or an error code if it fails (which can be tested using LZ4F_isError()) * A successful call to LZ4F_compressEnd() makes `cctx` available again for another compression task. diff --git a/tests/frametest.c b/tests/frametest.c index a99728f..fd07377 100644 --- a/tests/frametest.c +++ b/tests/frametest.c @@ -204,6 +204,13 @@ int basicTests(U32 seed, double compressibility) FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState); crcOrig = XXH64(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); + /* LZ4F_compressBound() : special case : srcSize == 0 */ + DISPLAYLEVEL(3, "LZ4F_compressBound(0) = "); + { size_t const cBound = LZ4F_compressBound(0, NULL); + if (cBound < 64 KB) goto _output_error; + DISPLAYLEVEL(3, " %u \n", (U32)cBound); + } + /* Special case : null-content frame */ testSize = 0; DISPLAYLEVEL(3, "LZ4F_compressFrame, compress null content : \n"); From bc0839c5df185129e899640e77a05f1c8fa595f6 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 21 Dec 2016 15:00:43 +0100 Subject: [PATCH 4/8] minor fix for travis-install test --- Makefile | 4 ++-- NEWS | 5 +++-- contrib/cmake_unofficial/.gitignore | 2 ++ 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 647ab4e..1432e6b 100644 --- a/Makefile +++ b/Makefile @@ -99,10 +99,10 @@ uninstall: @$(MAKE) -C $(PRGDIR) $@ travis-install: - $(MAKE) install PREFIX=~/install_test_dir + $(MAKE) -j1 install PREFIX=~/install_test_dir test: - $(MAKE) -C $(TESTDIR) test + $(MAKE) -C $(TESTDIR) $@ clangtest: clean clang -v diff --git a/NEWS b/NEWS index ff62ca8..e2f7666 100644 --- a/NEWS +++ b/NEWS @@ -1,8 +1,9 @@ v1.7.5 -cli : fix minor notification when using -r recursive mode lz4cat : fix : works with relative path (#284) and stdin (#285) (reported by @beiDei8z) +cli : fix minor notification when using -r recursive mode +API : lz4frame : LZ4F_frameBound(0) gives upper bound of *flush() and *End() operations (#290, #280) doc : markdown version of man page, by Takayuki Matsuoka (#279) -build : Makefile : fix concurrency lib+exe build (#277) +build : Makefile : fix make lib+exe concurrency (#277) v1.7.4.2 fix : Makefile : release build compatible with PIE and customized compilation directives provided through environment variables (#274, reported by Antoine Martin) diff --git a/contrib/cmake_unofficial/.gitignore b/contrib/cmake_unofficial/.gitignore index 0f81929..d39505d 100644 --- a/contrib/cmake_unofficial/.gitignore +++ b/contrib/cmake_unofficial/.gitignore @@ -5,3 +5,5 @@ CMakeFiles *.cmake Makefile liblz4.pc +lz4c +install_manifest.txt From 70526a11e033a640fb9e7ae80fe75b465d484bf0 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 21 Dec 2016 15:33:53 +0100 Subject: [PATCH 5/8] fixed lib/clean --- lib/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/Makefile b/lib/Makefile index 8426330..701dc07 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -107,7 +107,8 @@ endif liblz4: $(LIBLZ4) clean: - @$(RM) core *.o *.a *.$(SHARED_EXT) liblz4.pc dll/liblz4.dll dll/liblz4.lib + @$(RM) core *.o liblz4.pc dll/liblz4.dll dll/liblz4.lib + @$(RM) *.a *.$(SHARED_EXT) *.$(SHARED_EXT_MAJOR) *.$(SHARED_EXT_VER) @echo Cleaning library completed From 52cac9a97342641315c76cfb861206d6acd631a8 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 22 Dec 2016 11:41:05 +0100 Subject: [PATCH 6/8] updated a few macros names --- lib/lz4frame.c | 14 +++++++------- lib/lz4hc.c | 4 ++-- lib/lz4hc.h | 8 ++++---- programs/bench.c | 6 +++--- programs/lz4cli.c | 2 +- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/lib/lz4frame.c b/lib/lz4frame.c index 626ac98..a0a625b 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -329,7 +329,7 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, const void* srcBu if (prefs.frameInfo.contentSize != 0) prefs.frameInfo.contentSize = (U64)srcSize; /* auto-correct content size if selected (!=0) */ - if (prefs.compressionLevel < LZ4HC_MIN_CLEVEL) { + if (prefs.compressionLevel < LZ4HC_CLEVEL_MIN) { cctxI.lz4CtxPtr = &lz4ctx; cctxI.lz4CtxLevel = 1; } @@ -356,7 +356,7 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, const void* srcBu if (LZ4F_isError(tailSize)) return tailSize; dstPtr += tailSize; } - if (prefs.compressionLevel >= LZ4HC_MIN_CLEVEL) /* no allocation done with lz4 fast */ + if (prefs.compressionLevel >= LZ4HC_CLEVEL_MIN) /* no allocation done with lz4 fast */ FREEMEM(cctxI.lz4CtxPtr); return (dstPtr - dstStart); @@ -424,10 +424,10 @@ size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacit cctxPtr->prefs = *preferencesPtr; /* ctx Management */ - { U32 const tableID = (cctxPtr->prefs.compressionLevel < LZ4HC_MIN_CLEVEL) ? 1 : 2; /* 0:nothing ; 1:LZ4 table ; 2:HC tables */ + { U32 const tableID = (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) ? 1 : 2; /* 0:nothing ; 1:LZ4 table ; 2:HC tables */ if (cctxPtr->lz4CtxLevel < tableID) { FREEMEM(cctxPtr->lz4CtxPtr); - if (cctxPtr->prefs.compressionLevel < LZ4HC_MIN_CLEVEL) + if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) cctxPtr->lz4CtxPtr = (void*)LZ4_createStream(); else cctxPtr->lz4CtxPtr = (void*)LZ4_createStreamHC(); @@ -452,7 +452,7 @@ size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacit cctxPtr->tmpIn = cctxPtr->tmpBuff; cctxPtr->tmpInSize = 0; XXH32_reset(&(cctxPtr->xxh), 0); - if (cctxPtr->prefs.compressionLevel < LZ4HC_MIN_CLEVEL) + if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) LZ4_resetStream((LZ4_stream_t*)(cctxPtr->lz4CtxPtr)); else LZ4_resetStreamHC((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), cctxPtr->prefs.compressionLevel); @@ -533,7 +533,7 @@ static int LZ4F_localLZ4_compressHC_limitedOutput_continue(void* ctx, const char static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int level) { - if (level < LZ4HC_MIN_CLEVEL) { + if (level < LZ4HC_CLEVEL_MIN) { if (blockMode == LZ4F_blockIndependent) return LZ4F_localLZ4_compress_limitedOutput_withState; return LZ4F_localLZ4_compress_limitedOutput_continue; } @@ -543,7 +543,7 @@ static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int lev static int LZ4F_localSaveDict(LZ4F_cctx_t* cctxPtr) { - if (cctxPtr->prefs.compressionLevel < LZ4HC_MIN_CLEVEL) + if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) return LZ4_saveDict ((LZ4_stream_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB); return LZ4_saveDictHC ((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB); } diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 44a1340..ae245b5 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -497,7 +497,7 @@ static int LZ4HC_compress_generic ( limitedOutput_directive limit ) { - if (compressionLevel < 1) compressionLevel = LZ4HC_DEFAULT_CLEVEL; + if (compressionLevel < 1) compressionLevel = LZ4HC_CLEVEL_DEFAULT; if (compressionLevel > 9) { switch (compressionLevel) { case 10: return LZ4HC_compress_hashChain(ctx, source, dest, inputSize, maxOutputSize, 1 << (16-1), limit); @@ -574,7 +574,7 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock) { - if (ctxPtr->compressionLevel >= LZ4HC_MIN_CLEVEL_OPT) + if (ctxPtr->compressionLevel >= LZ4HC_CLEVEL_OPT_MIN) LZ4HC_updateBinTree(ctxPtr, ctxPtr->end - MFLIMIT, ctxPtr->end - LASTLITERALS); else if (ctxPtr->end >= ctxPtr->base + 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3); /* Referencing remaining dictionary content */ diff --git a/lib/lz4hc.h b/lib/lz4hc.h index 7ce5958..ad731d9 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -44,10 +44,10 @@ extern "C" { /* --- Useful constants --- */ -#define LZ4HC_MIN_CLEVEL 3 -#define LZ4HC_DEFAULT_CLEVEL 9 -#define LZ4HC_MIN_CLEVEL_OPT 11 -#define LZ4HC_MAX_CLEVEL 12 +#define LZ4HC_CLEVEL_MIN 3 +#define LZ4HC_CLEVEL_DEFAULT 9 +#define LZ4HC_CLEVEL_OPT_MIN 11 +#define LZ4HC_CLEVEL_MAX 12 /*-************************************ diff --git a/programs/bench.c b/programs/bench.c index 434da8b..406adcf 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -170,7 +170,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, UTIL_initTimer(&ticksPerSecond); /* Init */ - if (cLevel < LZ4HC_MIN_CLEVEL) cfunctionId = 0; else cfunctionId = 1; + if (cLevel < LZ4HC_CLEVEL_MIN) cfunctionId = 0; else cfunctionId = 1; switch (cfunctionId) { #ifdef COMPRESSOR0 @@ -494,8 +494,8 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, { double const compressibility = (double)g_compressibilityDefault / 100; - if (cLevel > LZ4HC_MAX_CLEVEL) cLevel = LZ4HC_MAX_CLEVEL; - if (cLevelLast > LZ4HC_MAX_CLEVEL) cLevelLast = LZ4HC_MAX_CLEVEL; + if (cLevel > LZ4HC_CLEVEL_MAX) cLevel = LZ4HC_CLEVEL_MAX; + if (cLevelLast > LZ4HC_CLEVEL_MAX) cLevelLast = LZ4HC_CLEVEL_MAX; if (cLevelLast < cLevel) cLevelLast = cLevel; if (cLevelLast > cLevel) DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast); diff --git a/programs/lz4cli.c b/programs/lz4cli.c index 5bd06d9..80d84a1 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -208,7 +208,7 @@ static int usage_longhelp(const char* exeName) DISPLAY( "Compression levels : \n"); DISPLAY( "---------------------\n"); DISPLAY( "-0 ... -2 => Fast compression, all identicals\n"); - DISPLAY( "-3 ... -%d => High compression; higher number == more compression but slower\n", LZ4HC_MAX_CLEVEL); + DISPLAY( "-3 ... -%d => High compression; higher number == more compression but slower\n", LZ4HC_CLEVEL_MAX); DISPLAY( "\n"); DISPLAY( "stdin, stdout and the console : \n"); DISPLAY( "--------------------------------\n"); From 7eb16d97d238e2b98f1e6453ed6ad64058cb99c1 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 22 Dec 2016 11:50:21 +0100 Subject: [PATCH 7/8] updated NEWS --- NEWS | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS b/NEWS index e2f7666..bb77e04 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,5 @@ v1.7.5 +lz4hc : new high compression mode : levels 10-12 compress more and slower, by @inikep. lz4cat : fix : works with relative path (#284) and stdin (#285) (reported by @beiDei8z) cli : fix minor notification when using -r recursive mode API : lz4frame : LZ4F_frameBound(0) gives upper bound of *flush() and *End() operations (#290, #280) From 19c0f21b000ababf11cb38e0d6154742f5e1cd83 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 22 Dec 2016 18:02:09 +0100 Subject: [PATCH 8/8] updated Makefile : lz4 no longer recompiled when already up-to-date --- NEWS | 2 +- lib/lz4opt.h | 59 ++++++++++++++++++++++++----------------------- programs/Makefile | 29 +++++++++++++---------- 3 files changed, 48 insertions(+), 42 deletions(-) diff --git a/NEWS b/NEWS index bb77e04..d149128 100644 --- a/NEWS +++ b/NEWS @@ -4,7 +4,7 @@ lz4cat : fix : works with relative path (#284) and stdin (#285) (reported by @be cli : fix minor notification when using -r recursive mode API : lz4frame : LZ4F_frameBound(0) gives upper bound of *flush() and *End() operations (#290, #280) doc : markdown version of man page, by Takayuki Matsuoka (#279) -build : Makefile : fix make lib+exe concurrency (#277) +build : Makefile : fix make -jX lib+exe concurrency (#277) v1.7.4.2 fix : Makefile : release build compatible with PIE and customized compilation directives provided through environment variables (#274, reported by Antoine Martin) diff --git a/lib/lz4opt.h b/lib/lz4opt.h index f487edf..94b38e0 100644 --- a/lib/lz4opt.h +++ b/lib/lz4opt.h @@ -2,7 +2,7 @@ lz4opt.h - Optimal Mode of LZ4 Copyright (C) 2015-2016, Przemyslaw Skibinski Note : this file is intended to be included within lz4hc.c - + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without @@ -33,9 +33,9 @@ - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ -#define LZ4_LOG_PARSER(fmt, ...) //printf(fmt, __VA_ARGS__) -#define LZ4_LOG_PRICE(fmt, ...) //printf(fmt, __VA_ARGS__) -#define LZ4_LOG_ENCODE(fmt, ...) //printf(fmt, __VA_ARGS__) +#define LZ4_LOG_PARSER(fmt, ...) //printf(fmt, __VA_ARGS__) +#define LZ4_LOG_PRICE(fmt, ...) //printf(fmt, __VA_ARGS__) +#define LZ4_LOG_ENCODE(fmt, ...) //printf(fmt, __VA_ARGS__) #define LZ4_OPT_NUM (1<<12) @@ -56,23 +56,24 @@ typedef struct } LZ4HC_optimal_t; -FORCE_INLINE size_t LZ4HC_GetLiteralsPrice(size_t litlen) +/* price in bits */ +FORCE_INLINE size_t LZ4HC_literalsPrice(size_t litlen) { size_t price = 8*litlen; - if (litlen >= (int)RUN_MASK) price+=8*(1+(litlen-RUN_MASK)/255); + if (litlen >= (size_t)RUN_MASK) price+=8*(1+(litlen-RUN_MASK)/255); return price; } -FORCE_INLINE size_t LZ4HC_get_price(size_t litlen, size_t mlen) +/* requires mlen >= MINMATCH */ +FORCE_INLINE size_t LZ4HC_sequencePrice(size_t litlen, size_t mlen) { size_t price = 16 + 8; /* 16-bit offset + token */ - price += 8*litlen; - if (litlen >= (int)RUN_MASK) price+=8*(1+(litlen-RUN_MASK)/255); + price += LZ4HC_literalsPrice(litlen); mlen -= MINMATCH; - if (mlen >= (int)ML_MASK) price+=8*(1+(mlen-ML_MASK)/255); + if (mlen >= (size_t)ML_MASK) price+=8*(1+(mlen-ML_MASK)/255); return price; } @@ -103,7 +104,7 @@ FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches ( U32 matchIndex; size_t matchLength = 0; U32* HashPos; - + if (ip + MINMATCH > iHighLimit) return 1; /* HC4 match finder */ @@ -164,7 +165,7 @@ FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches ( if (matchNum) *matchNum = mnum; /* if (best_mlen > 8) return best_mlen-8; */ if (!matchNum) return 1; - return 1; + return 1; } @@ -258,7 +259,7 @@ static int LZ4HC_compress_optimal ( LZ4_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-source), matches[i].len, matches[i].off, best_mlen, last_pos); while (mlen <= best_mlen) { litlen = 0; - price = LZ4HC_get_price(llen + litlen, mlen) - LZ4HC_GetLiteralsPrice(llen); + price = LZ4HC_sequencePrice(llen + litlen, mlen) - LZ4HC_literalsPrice(llen); SET_PRICE(mlen, mlen, matches[i].off, litlen, price); mlen++; } @@ -274,18 +275,18 @@ static int LZ4HC_compress_optimal ( if (opt[cur-1].mlen == 1) { litlen = opt[cur-1].litlen + 1; if (cur != litlen) { - price = opt[cur - litlen].price + LZ4HC_GetLiteralsPrice(litlen); + price = opt[cur - litlen].price + LZ4HC_literalsPrice(litlen); LZ4_LOG_PRICE("%d: TRY1 opt[%d].price=%d price=%d cur=%d litlen=%d\n", (int)(inr-source), cur - litlen, opt[cur - litlen].price, price, cur, litlen); } else { - price = LZ4HC_GetLiteralsPrice(llen + litlen) - LZ4HC_GetLiteralsPrice(llen); + price = LZ4HC_literalsPrice(llen + litlen) - LZ4HC_literalsPrice(llen); LZ4_LOG_PRICE("%d: TRY2 price=%d cur=%d litlen=%d llen=%d\n", (int)(inr-source), price, cur, litlen, llen); } } else { litlen = 1; - price = opt[cur - 1].price + LZ4HC_GetLiteralsPrice(litlen); - LZ4_LOG_PRICE("%d: TRY3 price=%d cur=%d litlen=%d litonly=%d\n", (int)(inr-source), price, cur, litlen, LZ4HC_GetLiteralsPrice(litlen)); + price = opt[cur - 1].price + LZ4HC_literalsPrice(litlen); + LZ4_LOG_PRICE("%d: TRY3 price=%d cur=%d litlen=%d litonly=%d\n", (int)(inr-source), price, cur, litlen, LZ4HC_literalsPrice(litlen)); } - + mlen = 1; best_mlen = 0; LZ4_LOG_PARSER("%d: TRY price=%d opt[%d].price=%d\n", (int)(inr-source), price, cur, opt[cur].price); @@ -293,7 +294,7 @@ static int LZ4HC_compress_optimal ( SET_PRICE(cur, mlen, best_mlen, litlen, price); if (cur == last_pos || inr >= mflimit) break; - LZ4_LOG_PARSER("%d: CURRENT price[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(inr-source), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen); + LZ4_LOG_PARSER("%d: CURRENT price[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(inr-source), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen); match_num = LZ4HC_BinTree_GetAllMatches(ctx, inr, matchlimit, MINMATCH-1, matches, fullUpdate); LZ4_LOG_PARSER("%d: LZ4HC_BinTree_GetAllMatches match_num=%d\n", (int)(inr-source), match_num); @@ -317,12 +318,12 @@ static int LZ4HC_compress_optimal ( litlen = opt[cur2].litlen; if (cur2 != litlen) - price = opt[cur2 - litlen].price + LZ4HC_get_price(litlen, mlen); + price = opt[cur2 - litlen].price + LZ4HC_sequencePrice(litlen, mlen); else - price = LZ4HC_get_price(llen + litlen, mlen) - LZ4HC_GetLiteralsPrice(llen); + price = LZ4HC_sequencePrice(llen + litlen, mlen) - LZ4HC_literalsPrice(llen); } else { litlen = 0; - price = opt[cur2].price + LZ4HC_get_price(litlen, mlen); + price = opt[cur2].price + LZ4HC_sequencePrice(litlen, mlen); } LZ4_LOG_PARSER("%d: Found2 mlen=%d best_mlen=%d off=%d price=%d litlen=%d price[%d]=%d\n", (int)(inr-source), mlen, best_mlen, matches[i].off, price, litlen, cur - litlen, opt[cur - litlen].price); @@ -340,31 +341,31 @@ static int LZ4HC_compress_optimal ( encode: /* cur, last_pos, best_mlen, best_off have to be set */ for (i = 1; i <= last_pos; i++) { - LZ4_LOG_PARSER("%d: price[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-source+i), i, last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen); + LZ4_LOG_PARSER("%d: price[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-source+i), i, last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen); } - LZ4_LOG_PARSER("%d: cur=%d/%d best_mlen=%d best_off=%d\n", (int)(ip-source+cur), cur, last_pos, best_mlen, best_off); + LZ4_LOG_PARSER("%d: cur=%d/%d best_mlen=%d best_off=%d\n", (int)(ip-source+cur), cur, last_pos, best_mlen, best_off); opt[0].mlen = 1; while (1) { mlen = opt[cur].mlen; offset = opt[cur].off; - opt[cur].mlen = (int)best_mlen; + opt[cur].mlen = (int)best_mlen; opt[cur].off = (int)best_off; best_mlen = mlen; best_off = offset; if (mlen > cur) break; cur -= mlen; } - + for (i = 0; i <= last_pos;) { - LZ4_LOG_PARSER("%d: price2[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-source+i), i, last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen); + LZ4_LOG_PARSER("%d: price2[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-source+i), i, last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen); i += opt[i].mlen; } cur = 0; while (cur < last_pos) { - LZ4_LOG_PARSER("%d: price3[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-source+cur), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen); + LZ4_LOG_PARSER("%d: price3[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-source+cur), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen); mlen = opt[cur].mlen; if (mlen == 1) { ip++; cur++; continue; } offset = opt[cur].off; @@ -374,7 +375,7 @@ encode: /* cur, last_pos, best_mlen, best_off have to be set */ res = LZ4HC_encodeSequence(&ip, &op, &anchor, (int)mlen, ip - offset, limit, oend); LZ4_LOG_ENCODE("out=%d\n", (int)((char*)op - dest)); - if (res) return 0; + if (res) return 0; LZ4_LOG_PARSER("%d: offset=%d\n", (int)(ip-source), offset); } diff --git a/programs/Makefile b/programs/Makefile index 74a2440..6f4c8c2 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -43,7 +43,9 @@ PREFIX ?= /usr/local BINDIR := $(PREFIX)/bin MANDIR := $(PREFIX)/share/man/man1 LZ4DIR := ../lib -VOID := /dev/null + +SRCFILES := $(wildcard $(LZ4DIR)/*.c) $(wildcard *.c) +OBJFILES := $(patsubst %.c,%.o,$(SRCFILES)) CPPFLAGS+= -I$(LZ4DIR) -DXXH_NAMESPACE=LZ4_ CFLAGS ?= -O3 @@ -60,33 +62,36 @@ MD2ROFF_FLAGS = --roff --warnings --manual="User Commands" --organization="lz4 # Define *.exe as extension for Windows systems ifneq (,$(filter Windows%,$(OS))) -EXT =.exe +VOID := nul +EXT :=.exe else -EXT = +VOID := /dev/null +EXT := endif default: lz4-release -lz4-release: $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/lz4frame.o $(LZ4DIR)/xxhash.o bench.o lz4io.o lz4cli.o datagen.o - $(CC) $(FLAGS) $^ -o lz4$(EXT) - all: lz4 lz4c all32: CFLAGS+=-m32 all32: all lz4: CFLAGS += $(DEBUGFLAGS) -lz4: lz4-release +lz4: $(OBJFILES) + $(CC) $(FLAGS) $^ -o $@$(EXT) -lz4c: CFLAGS += $(DEBUGFLAGS) -lz4c : $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/lz4frame.o $(LZ4DIR)/xxhash.o bench.o lz4io.o lz4cli.c datagen.o +lz4-release: DEBUGFLAGS= +lz4-release: lz4 + +lz4c : CFLAGS += $(DEBUGFLAGS) +lz4c : $(SRCFILES) $(CC) $(FLAGS) -DENABLE_LZ4C_LEGACY_OPTIONS $^ -o $@$(EXT) -lz4c32: CFLAGS+=-m32 -lz4c32: lz4 - @cp lz4$(EXT) lz4c32$(EXT) +lz4c32: CFLAGS += -m32 $(DEBUGFLAGS) +lz4c32: $(SRCFILES) + $(CC) $(FLAGS) $^ -o $@$(EXT) clean: @$(MAKE) -C $(LZ4DIR) $@ > $(VOID)