From c1864f69e87d68fe79420c9303ecf97eb66bc7c7 Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 2 Sep 2016 21:50:57 +0200 Subject: [PATCH 1/3] fix mingw64 compilation warnings --- .gitignore | 3 +++ programs/bench.c | 4 ++++ programs/lz4cli.c | 2 +- programs/lz4io.c | 3 +++ 4 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index a715f8a..d1ce131 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,6 @@ lz4 # Visual solution files *.suo *.user + +# Directories +_codelite/ diff --git a/programs/bench.c b/programs/bench.c index f09c912..b03d562 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -39,6 +39,10 @@ # define _LARGEFILE64_SOURCE #endif +#if defined(__MINGW32__) && !defined(_POSIX_SOURCE) +# define _POSIX_SOURCE 1 /* disable %llu warnings with MinGW on Windows */ +#endif + /*-************************************ * Includes diff --git a/programs/lz4cli.c b/programs/lz4cli.c index 12d48d1..59312a0 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -88,7 +88,7 @@ ******************************/ #define COMPRESSOR_NAME "LZ4 command line interface" #ifndef LZ4_VERSION -# define LZ4_VERSION "r128" +# define LZ4_VERSION "r132" #endif #define AUTHOR "Yann Collet" #define WELCOME_MESSAGE "*** %s %i-bits %s, by %s (%s) ***\n", COMPRESSOR_NAME, (int)(sizeof(void*)*8), LZ4_VERSION, AUTHOR, __DATE__ diff --git a/programs/lz4io.c b/programs/lz4io.c index e4823dc..bc1213a 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -42,6 +42,9 @@ #define _LARGE_FILES /* Large file support on 32-bits AIX */ #define _FILE_OFFSET_BITS 64 /* Large file support on 32-bits unix */ +#if defined(__MINGW32__) && !defined(_POSIX_SOURCE) +# define _POSIX_SOURCE 1 /* disable %llu warnings with MinGW on Windows */ +#endif /***************************** * Includes From 2cf32ef0f107d18c9779283b8bd1d56f0517dd39 Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 2 Sep 2016 22:59:18 +0200 Subject: [PATCH 2/3] -b# and -e# options from zstd --- lib/lz4frame.c | 15 +++++++-------- lib/lz4hc.c | 7 ++----- lib/lz4hc.h | 8 ++++++-- programs/bench.c | 45 ++++++++++++++++++++++++++++++++------------- programs/bench.h | 3 ++- programs/lz4cli.c | 40 +++++++++++++++++++++++++++++----------- 6 files changed, 78 insertions(+), 40 deletions(-) diff --git a/lib/lz4frame.c b/lib/lz4frame.c index 2e42dcc..1dc2bca 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -150,7 +150,6 @@ static void LZ4F_writeLE64 (BYTE* dstPtr, U64 value64) static const size_t minFHSize = 7; static const size_t maxFHSize = 15; static const size_t BHSize = 4; -static const int minHClevel = 3; /*-************************************ @@ -306,7 +305,7 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuf if (prefs.frameInfo.contentSize != 0) prefs.frameInfo.contentSize = (U64)srcSize; /* auto-correct content size if selected (!=0) */ - if (prefs.compressionLevel < (int)minHClevel) { + if (prefs.compressionLevel < LZ4HC_MIN_CLEVEL) { cctxI.lz4CtxPtr = &lz4ctx; cctxI.lz4CtxLevel = 1; } @@ -333,7 +332,7 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuf if (LZ4F_isError(errorCode)) return errorCode; dstPtr += errorCode; - if (prefs.compressionLevel >= (int)minHClevel) /* no allocation necessary with lz4 fast */ + if (prefs.compressionLevel >= LZ4HC_MIN_CLEVEL) /* no allocation necessary with lz4 fast */ FREEMEM(cctxI.lz4CtxPtr); return (dstPtr - dstStart); @@ -404,10 +403,10 @@ size_t LZ4F_compressBegin(LZ4F_compressionContext_t compressionContext, void* ds cctxPtr->prefs = *preferencesPtr; /* ctx Management */ - { U32 const tableID = (cctxPtr->prefs.compressionLevel < minHClevel) ? 1 : 2; /* 0:nothing ; 1:LZ4 table ; 2:HC tables */ + { U32 const tableID = (cctxPtr->prefs.compressionLevel < LZ4HC_MIN_CLEVEL) ? 1 : 2; /* 0:nothing ; 1:LZ4 table ; 2:HC tables */ if (cctxPtr->lz4CtxLevel < tableID) { FREEMEM(cctxPtr->lz4CtxPtr); - if (cctxPtr->prefs.compressionLevel < minHClevel) + if (cctxPtr->prefs.compressionLevel < LZ4HC_MIN_CLEVEL) cctxPtr->lz4CtxPtr = (void*)LZ4_createStream(); else cctxPtr->lz4CtxPtr = (void*)LZ4_createStreamHC(); @@ -432,7 +431,7 @@ size_t LZ4F_compressBegin(LZ4F_compressionContext_t compressionContext, void* ds cctxPtr->tmpIn = cctxPtr->tmpBuff; cctxPtr->tmpInSize = 0; XXH32_reset(&(cctxPtr->xxh), 0); - if (cctxPtr->prefs.compressionLevel < minHClevel) + if (cctxPtr->prefs.compressionLevel < LZ4HC_MIN_CLEVEL) LZ4_resetStream((LZ4_stream_t*)(cctxPtr->lz4CtxPtr)); else LZ4_resetStreamHC((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), cctxPtr->prefs.compressionLevel); @@ -525,7 +524,7 @@ static int LZ4F_localLZ4_compressHC_limitedOutput_continue(void* ctx, const char static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int level) { - if (level < minHClevel) { + if (level < LZ4HC_MIN_CLEVEL) { if (blockMode == LZ4F_blockIndependent) return LZ4F_localLZ4_compress_limitedOutput_withState; return LZ4F_localLZ4_compress_limitedOutput_continue; } @@ -535,7 +534,7 @@ static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int lev static int LZ4F_localSaveDict(LZ4F_cctx_t* cctxPtr) { - if (cctxPtr->prefs.compressionLevel < minHClevel) + if (cctxPtr->prefs.compressionLevel < LZ4HC_MIN_CLEVEL) return LZ4_saveDict ((LZ4_stream_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB); return LZ4_saveDictHC ((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB); } diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 80bfa39..8bcebc8 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -37,7 +37,6 @@ /* ************************************* * Tuning Parameter ***************************************/ -static const int LZ4HC_compressionLevel_default = 9; /*! * HEAPMODE : @@ -86,8 +85,6 @@ static const int LZ4HC_compressionLevel_default = 9; #define OPTIMAL_ML (int)((ML_MASK-1)+MINMATCH) -static const int g_maxCompressionLevel = 16; - /************************************** * Local Types @@ -371,8 +368,8 @@ static int LZ4HC_compress_generic ( /* init */ - if (compressionLevel > g_maxCompressionLevel) compressionLevel = g_maxCompressionLevel; - if (compressionLevel < 1) compressionLevel = LZ4HC_compressionLevel_default; + if (compressionLevel > LZ4HC_MAX_CLEVEL) compressionLevel = LZ4HC_MAX_CLEVEL; + if (compressionLevel < 1) compressionLevel = LZ4HC_DEFAULT_CLEVEL; maxNbAttempts = 1 << (compressionLevel-1); ctx->end += inputSize; diff --git a/lib/lz4hc.h b/lib/lz4hc.h index fce2213..75eed19 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -45,6 +45,10 @@ extern "C" { #include /* size_t */ +#define LZ4HC_MIN_CLEVEL 3 +#define LZ4HC_DEFAULT_CLEVEL 9 +#define LZ4HC_MAX_CLEVEL 16 + /*-************************************ * Block Compression **************************************/ @@ -54,9 +58,9 @@ LZ4_compress_HC() : Compression success is guaranteed if `dst` buffer is sized to handle worst circumstances (data not compressible) Worst size evaluation is provided by function LZ4_compressBound() (see "lz4.h") `srcSize` : Max supported value is LZ4_MAX_INPUT_SIZE (see "lz4.h") - `compressionLevel` : Recommended values are between 4 and 9, although any value between 0 and 16 will work. + `compressionLevel` : Recommended values are between 4 and 9, although any value between 0 and LZ4HC_MAX_CLEVEL will work. 0 means "use default value" (see lz4hc.c). - Values >16 behave the same as 16. + Values >LZ4HC_MAX_CLEVEL behave the same as 16. @return : the number of bytes written into buffer 'dst' or 0 if compression fails. */ diff --git a/programs/bench.c b/programs/bench.c index b03d562..b56bac2 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -201,7 +201,7 @@ static U64 BMK_GetFileSize(const char* infilename) * Public function **********************************************************/ -int BMK_benchFiles(const char** fileNamesTable, int nbFiles, int cLevel) +int BMK_benchLevel(const char** fileNamesTable, int nbFiles, int cLevel) { int fileIdx=0; char* orig_buff; @@ -214,7 +214,7 @@ int BMK_benchFiles(const char** fileNamesTable, int nbFiles, int cLevel) double totald = 0.; /* Init */ - if (cLevel <= 2) cfunctionId = 0; else cfunctionId = 1; + if (cLevel < LZ4HC_MIN_CLEVEL) cfunctionId = 0; else cfunctionId = 1; switch (cfunctionId) { #ifdef COMPRESSOR0 @@ -228,7 +228,7 @@ int BMK_benchFiles(const char** fileNamesTable, int nbFiles, int cLevel) /* Loop for each file */ while (fileIdx16) inFileName += strlen(inFileName)-16; /* can only display 16 characters */ readSize = fread(orig_buff, 1, benchedSize, inFile); fclose(inFile); @@ -308,7 +309,7 @@ int BMK_benchFiles(const char** fileNamesTable, int nbFiles, int cLevel) unsigned chunkNb; /* Compression */ - DISPLAY("%1i-%-14.14s : %9i ->\r", loopNb, inFileName, (int)benchedSize); + DISPLAY("%2i#%1i-%-14.14s : %9i ->\r", cLevel, loopNb, inFileName, (int)benchedSize); { size_t i; for (i=0; i %9i (%5.2f%%),%7.1f MB/s\r", - loopNb, inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / (fastestC / CLOCKS_PER_SEC) / 1000000); + DISPLAY("%2i#%1i-%-14.14s : %9i -> %9i (%5.2f%%),%7.1f MB/s\r", + cLevel, loopNb, inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / (fastestC / CLOCKS_PER_SEC) / 1000000); /* Decompression */ { size_t i; for (i=0; i %9i (%5.2f%%),%7.1f MB/s ,%7.1f MB/s \r", - loopNb, inFileName, (int)benchedSize, (int)cSize, ratio, + DISPLAY("%2i#%1i-%-14.14s : %9i -> %9i (%5.2f%%),%7.1f MB/s ,%7.1f MB/s \r", + cLevel, loopNb, inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / (fastestC / CLOCKS_PER_SEC) / 1000000, (double)benchedSize / (fastestD / CLOCKS_PER_SEC) / 1000000 ); /* CRC Checking */ @@ -357,12 +358,12 @@ int BMK_benchFiles(const char** fileNamesTable, int nbFiles, int cLevel) if (crcOrig==crcCheck) { if (ratio < 100.) - DISPLAY("%-16.16s : %9i -> %9i (%5.2f%%),%7.1f MB/s ,%7.1f MB/s \n", - inFileName, (int)benchedSize, (int)cSize, ratio, + DISPLAY("%2i#%-16.16s : %9i -> %9i (%5.2f%%),%7.1f MB/s ,%7.1f MB/s \n", + cLevel, inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / (fastestC / CLOCKS_PER_SEC) / 1000000, (double)benchedSize / (fastestD / CLOCKS_PER_SEC) / 1000000 ); else - DISPLAY("%-16.16s : %9i -> %9i (%5.1f%%),%7.1f MB/s ,%7.1f MB/s \n", - inFileName, (int)benchedSize, (int)cSize, ratio, + DISPLAY("%2i#%-16.16s : %9i -> %9i (%5.1f%%),%7.1f MB/s ,%7.1f MB/s \n", + cLevel, inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / (fastestC / CLOCKS_PER_SEC) / 1000000, (double)benchedSize / (fastestD / CLOCKS_PER_SEC) / 1000000 ); } totals += benchedSize; @@ -377,7 +378,7 @@ int BMK_benchFiles(const char** fileNamesTable, int nbFiles, int cLevel) } if (nbFiles > 1) - DISPLAY("%-16.16s :%10llu ->%10llu (%5.2f%%), %6.1f MB/s , %6.1f MB/s\n", " TOTAL", + DISPLAY("%2i#%-16.16s :%10llu ->%10llu (%5.2f%%), %6.1f MB/s , %6.1f MB/s\n", cLevel, " TOTAL", (long long unsigned)totals, (long long unsigned int)totalz, (double)totalz/(double)totals*100., (double)totals/(totalc/CLOCKS_PER_SEC)/1000000, (double)totals/(totald/CLOCKS_PER_SEC)/1000000); @@ -385,3 +386,21 @@ int BMK_benchFiles(const char** fileNamesTable, int nbFiles, int cLevel) return 0; } + + +int BMK_benchFiles(const char** fileNamesTable, int nbFiles, int cLevel, int cLevelLast) +{ + int i, res = 0; + + if (cLevel > LZ4HC_MAX_CLEVEL) cLevel = LZ4HC_MAX_CLEVEL; + if (cLevelLast > LZ4HC_MAX_CLEVEL) cLevelLast = LZ4HC_MAX_CLEVEL; + if (cLevelLast < cLevel) cLevelLast = cLevel; + + DISPLAY("Benchmarking levels from %d to %d\n", cLevel, cLevelLast); + for (i=cLevel; i<=cLevelLast; i++) { + res = BMK_benchLevel(fileNamesTable, nbFiles, i); + if (res != 0) break; + } + + return res; +} diff --git a/programs/bench.h b/programs/bench.h index c04fb17..1e3df78 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -24,7 +24,8 @@ /* Main function */ -int BMK_benchFiles(const char** fileNamesTable, int nbFiles, int cLevel); +int BMK_benchFiles(const char** fileNamesTable, int nbFiles, int cLevel, int cLevelLast); +int BMK_benchLevel(const char** fileNamesTable, int nbFiles, int cLevel); /* Set Parameters */ void BMK_setBlocksize(int bsize); diff --git a/programs/lz4cli.c b/programs/lz4cli.c index 59312a0..a583481 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -61,7 +61,7 @@ #include /* strcmp, strlen */ #include "bench.h" /* BMK_benchFile, BMK_SetNbIterations, BMK_SetBlocksize, BMK_SetPause */ #include "lz4io.h" /* LZ4IO_compressFilename, LZ4IO_decompressFilename, LZ4IO_compressMultipleFilenames */ - +#include "lz4hc.h" /* LZ4HC_DEFAULT_CLEVEL */ /**************************** * OS-specific Includes @@ -183,7 +183,9 @@ static int usage_advanced(void) DISPLAY( "--content-size : compressed frame includes original size (default:not present)\n"); DISPLAY( "--[no-]sparse : sparse mode (default:enabled on file, disabled on stdout)\n"); DISPLAY( "Benchmark arguments :\n"); - DISPLAY( " -b : benchmark file(s)\n"); + DISPLAY( "Benchmark arguments :\n"); + DISPLAY( " -b# : benchmark file(s), using # compression level (default : 1) \n"); + DISPLAY( " -e# : test all compression levels from -bX to # (default: 1)\n"); DISPLAY( " -i# : iteration loops [1-9](default : 3), benchmark mode only\n"); #if defined(ENABLE_LZ4C_LEGACY_OPTIONS) DISPLAY( "Legacy arguments :\n"); @@ -219,7 +221,7 @@ static int usage_longhelp(void) DISPLAY( "Compression levels : \n"); DISPLAY( "---------------------\n"); DISPLAY( "-0 ... -2 => Fast compression, all identicals\n"); - DISPLAY( "-3 ... -16 => High compression; higher number == more compression but slower\n"); + DISPLAY( "-3 ... -%d => High compression; higher number == more compression but slower\n", LZ4HC_MAX_CLEVEL); DISPLAY( "\n"); DISPLAY( "stdin, stdout and the console : \n"); DISPLAY( "--------------------------------\n"); @@ -272,10 +274,24 @@ static void waitEnter(void) } +/*! readU32FromChar() : + @return : unsigned integer value reach from input in `char` format + Will also modify `*stringPtr`, advancing it to position where it stopped reading. + Note : this function can overflow if result > MAX_UINT */ +static unsigned readU32FromChar(const char** stringPtr) +{ + unsigned result = 0; + while ((**stringPtr >='0') && (**stringPtr <='9')) + result *= 10, result += **stringPtr - '0', (*stringPtr)++ ; + return result; +} + + int main(int argc, const char** argv) { int i, - cLevel=0, + cLevel=1, + cLevelLast=1, decode=0, bench=0, legacy_format=0, @@ -350,16 +366,12 @@ int main(int argc, const char** argv) #endif /* ENABLE_LZ4C_LEGACY_OPTIONS */ if ((*argument>='0') && (*argument<='9')) { - cLevel = 0; - while ((*argument >= '0') && (*argument <= '9')) { - cLevel *= 10; - cLevel += *argument - '0'; - argument++; - } + cLevel = readU32FromChar(&argument); argument--; continue; } + switch(argument[0]) { /* Display help */ @@ -367,6 +379,12 @@ int main(int argc, const char** argv) case 'h': usage_advanced(); goto _cleanup; case 'H': usage_longhelp(); goto _cleanup; + case 'e': + argument++; + cLevelLast = readU32FromChar(&argument); + argument--; + break; + /* Compression (default) */ case 'z': forceCompress = 1; break; @@ -492,7 +510,7 @@ int main(int argc, const char** argv) /* Check if benchmark is selected */ if (bench) { - int bmkResult = BMK_benchFiles(inFileNames, ifnIdx, cLevel); + int bmkResult = BMK_benchFiles(inFileNames, ifnIdx, cLevel, cLevelLast); free((void*)inFileNames); return bmkResult; } From b3ae49c7bf3652a37ba42095b9f1177c0eff49e3 Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 2 Sep 2016 23:02:30 +0200 Subject: [PATCH 3/3] added for strlen() --- programs/bench.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/programs/bench.c b/programs/bench.c index b56bac2..81118c0 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -52,6 +52,8 @@ #include /* stat64 */ #include /* stat64 */ #include /* clock_t, clock, CLOCKS_PER_SEC */ +#include /* strlen */ + #include "lz4.h" #define COMPRESSOR0 LZ4_compress_local