diff --git a/Makefile b/Makefile index 2003da90..cea6cb60 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,7 @@ # ################################################################ # Version number -export VERSION := 0.3.5 +export VERSION := 0.3.6 PRGDIR = programs ZSTDDIR = lib diff --git a/NEWS b/NEWS index a5e18c12..0b635d80 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,6 @@ +v0.3.6 +small blocks params + v0.3.5 minor generic compression improvements diff --git a/images/CSpeed.png b/images/CSpeed.png index 338c27b2..97497101 100755 Binary files a/images/CSpeed.png and b/images/CSpeed.png differ diff --git a/lib/zstd.h b/lib/zstd.h index b13400ec..81437885 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -48,7 +48,7 @@ extern "C" { ***************************************/ #define ZSTD_VERSION_MAJOR 0 /* for breaking interface changes */ #define ZSTD_VERSION_MINOR 3 /* for new (non-breaking) interface capabilities */ -#define ZSTD_VERSION_RELEASE 5 /* for tweaks, bug-fixes, or development */ +#define ZSTD_VERSION_RELEASE 6 /* for tweaks, bug-fixes, or development */ #define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) unsigned ZSTD_versionNumber (void); diff --git a/lib/zstd_internal.h b/lib/zstd_internal.h index dffa1fe6..d606e23d 100644 --- a/lib/zstd_internal.h +++ b/lib/zstd_internal.h @@ -226,6 +226,14 @@ static const U32 g_searchStrength = 8; */ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const BYTE* literals, size_t offsetCode, size_t matchCode) { +#if 0 + static const BYTE* g_start = NULL; + if (g_start==NULL) g_start = literals; + if (literals - g_start == 8695) + printf("pos %6u : %3u literals & match %3u bytes at distance %6u \n", + (U32)(literals - g_start), (U32)litLength, (U32)matchCode+4, (U32)offsetCode); +#endif + /* copy Literals */ ZSTD_wildcopy(seqStorePtr->lit, literals, litLength); seqStorePtr->lit += litLength; diff --git a/lib/zstdhc.c b/lib/zstdhc.c index cfc098ac..fdf6dc95 100644 --- a/lib/zstdhc.c +++ b/lib/zstdhc.c @@ -111,7 +111,7 @@ size_t ZSTD_HC_freeCCtx(ZSTD_HC_CCtx* cctx) /** ZSTD_HC_validateParams correct params value to remain within authorized range optimize for srcSize if srcSize > 0 */ -void ZSTD_HC_validateParams(ZSTD_HC_parameters* params, size_t srcSize) +void ZSTD_HC_validateParams(ZSTD_HC_parameters* params, U64 srcSizeHint) { const U32 btPlus = (params->strategy == ZSTD_HC_btlazy2); @@ -120,9 +120,9 @@ void ZSTD_HC_validateParams(ZSTD_HC_parameters* params, size_t srcSize) if (params->windowLog < ZSTD_HC_WINDOWLOG_MIN) params->windowLog = ZSTD_HC_WINDOWLOG_MIN; /* correct params, to use less memory */ - if (srcSize > 0) + if ((srcSizeHint > 0) && (srcSizeHint < (1<windowLog > srcLog) params->windowLog = srcLog; } @@ -139,9 +139,10 @@ void ZSTD_HC_validateParams(ZSTD_HC_parameters* params, size_t srcSize) static size_t ZSTD_HC_resetCCtx_advanced (ZSTD_HC_CCtx* zc, - ZSTD_HC_parameters params) + ZSTD_HC_parameters params, + U64 srcSizeHint) { - ZSTD_HC_validateParams(¶ms, 0); + ZSTD_HC_validateParams(¶ms, srcSizeHint); /* reserve table memory */ { @@ -930,7 +931,7 @@ size_t ZSTD_HC_compressContinue (ZSTD_HC_CCtx* ctxPtr, if (ip != ctxPtr->end) { if (ctxPtr->end != NULL) - ZSTD_HC_resetCCtx_advanced(ctxPtr, ctxPtr->params); + ZSTD_HC_resetCCtx_advanced(ctxPtr, ctxPtr->params, srcSize); ctxPtr->base = ip; } @@ -941,22 +942,24 @@ size_t ZSTD_HC_compressContinue (ZSTD_HC_CCtx* ctxPtr, size_t ZSTD_HC_compressBegin_advanced(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, - const ZSTD_HC_parameters params) + const ZSTD_HC_parameters params, + U64 srcSizeHint) { size_t errorCode; if (maxDstSize < 4) return ERROR(dstSize_tooSmall); - errorCode = ZSTD_HC_resetCCtx_advanced(ctx, params); + errorCode = ZSTD_HC_resetCCtx_advanced(ctx, params, srcSizeHint); if (ZSTD_isError(errorCode)) return errorCode; MEM_writeLE32(dst, ZSTD_magicNumber); /* Write Header */ return 4; } -size_t ZSTD_HC_compressBegin(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, int compressionLevel) +size_t ZSTD_HC_compressBegin(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, int compressionLevel, U64 srcSizeHint) { + int tableID = ((srcSizeHint-1) > 128 KB); /* intentional underflow for 0 */ if (compressionLevel<=0) compressionLevel = 1; if (compressionLevel > ZSTD_HC_MAX_CLEVEL) compressionLevel = ZSTD_HC_MAX_CLEVEL; - return ZSTD_HC_compressBegin_advanced(ctx, dst, maxDstSize, ZSTD_HC_defaultParameters[compressionLevel]); + return ZSTD_HC_compressBegin_advanced(ctx, dst, maxDstSize, ZSTD_HC_defaultParameters[tableID][compressionLevel], srcSizeHint); } @@ -994,7 +997,7 @@ size_t ZSTD_HC_compress_advanced (ZSTD_HC_CCtx* ctx, } /* Header */ - oSize = ZSTD_HC_compressBegin_advanced(ctx, dst, maxDstSize, params); + oSize = ZSTD_HC_compressBegin_advanced(ctx, dst, maxDstSize, params, srcSize); if(ZSTD_isError(oSize)) return oSize; op += oSize; maxDstSize -= oSize; @@ -1016,9 +1019,10 @@ size_t ZSTD_HC_compress_advanced (ZSTD_HC_CCtx* ctx, size_t ZSTD_HC_compressCCtx (ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel) { + const int tableID = (srcSize > 128 KB); if (compressionLevel<=1) return ZSTD_compress(dst, maxDstSize, src, srcSize); /* fast mode */ if (compressionLevel > ZSTD_HC_MAX_CLEVEL) compressionLevel = ZSTD_HC_MAX_CLEVEL; - return ZSTD_HC_compress_advanced(ctx, dst, maxDstSize, src, srcSize, ZSTD_HC_defaultParameters[compressionLevel]); + return ZSTD_HC_compress_advanced(ctx, dst, maxDstSize, src, srcSize, ZSTD_HC_defaultParameters[tableID][compressionLevel]); } size_t ZSTD_HC_compress(void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel) diff --git a/lib/zstdhc_static.h b/lib/zstdhc_static.h index f96d4055..a08b00bf 100644 --- a/lib/zstdhc_static.h +++ b/lib/zstdhc_static.h @@ -83,14 +83,14 @@ size_t ZSTD_HC_compress_advanced (ZSTD_HC_CCtx* ctx, /** ZSTD_HC_validateParams correct params value to remain within authorized range - optimize for srcSize if srcSize > 0 */ -void ZSTD_HC_validateParams(ZSTD_HC_parameters* params, size_t srcSize); + srcSizeHint value is optional, select 0 if not known */ +void ZSTD_HC_validateParams(ZSTD_HC_parameters* params, U64 srcSizeHint); /* ************************************* * Streaming functions ***************************************/ -size_t ZSTD_HC_compressBegin(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, int compressionLevel); +size_t ZSTD_HC_compressBegin(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, int compressionLevel, U64 srcSizeHint); size_t ZSTD_HC_compressContinue(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); size_t ZSTD_HC_compressEnd(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize); @@ -99,7 +99,32 @@ size_t ZSTD_HC_compressEnd(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize); * Pre-defined compression levels ***************************************/ #define ZSTD_HC_MAX_CLEVEL 20 -static const ZSTD_HC_parameters ZSTD_HC_defaultParameters[ZSTD_HC_MAX_CLEVEL+1] = { +static const ZSTD_HC_parameters ZSTD_HC_defaultParameters[2][ZSTD_HC_MAX_CLEVEL+1] = { +{ /* for <= 128 KB */ + /* W, C, H, S, L, strat */ + { 17, 12, 12, 1, 4, ZSTD_HC_fast }, /* level 0 - never used */ + { 17, 12, 13, 1, 6, ZSTD_HC_fast }, /* level 1 */ + { 17, 15, 16, 1, 5, ZSTD_HC_fast }, /* level 2 */ + { 17, 16, 17, 1, 5, ZSTD_HC_fast }, /* level 3 */ + { 17, 13, 15, 2, 4, ZSTD_HC_greedy }, /* level 4 */ + { 17, 15, 17, 3, 4, ZSTD_HC_greedy }, /* level 5 */ + { 17, 14, 17, 3, 4, ZSTD_HC_lazy }, /* level 6 */ + { 17, 16, 17, 4, 4, ZSTD_HC_lazy }, /* level 7 */ + { 17, 16, 17, 4, 4, ZSTD_HC_lazy2 }, /* level 8 */ + { 17, 17, 16, 5, 4, ZSTD_HC_lazy2 }, /* level 9 */ + { 17, 17, 16, 6, 4, ZSTD_HC_lazy2 }, /* level 10 */ + { 17, 17, 16, 7, 4, ZSTD_HC_lazy2 }, /* level 11 */ + { 17, 17, 16, 8, 4, ZSTD_HC_lazy2 }, /* level 12 */ + { 17, 18, 16, 4, 4, ZSTD_HC_btlazy2 }, /* level 13 */ + { 17, 18, 16, 5, 4, ZSTD_HC_btlazy2 }, /* level 14 */ + { 17, 18, 16, 6, 4, ZSTD_HC_btlazy2 }, /* level 15 */ + { 17, 18, 16, 7, 4, ZSTD_HC_btlazy2 }, /* level 16 */ + { 17, 18, 16, 8, 4, ZSTD_HC_btlazy2 }, /* level 17 */ + { 17, 18, 16, 9, 4, ZSTD_HC_btlazy2 }, /* level 18 */ + { 17, 18, 16, 10, 4, ZSTD_HC_btlazy2 }, /* level 19 */ + { 17, 18, 18, 12, 4, ZSTD_HC_btlazy2 }, /* level 20 */ +}, +{ /* for > 128 KB */ /* W, C, H, S, L, strat */ { 18, 12, 12, 1, 4, ZSTD_HC_fast }, /* level 0 - never used */ { 18, 14, 14, 1, 7, ZSTD_HC_fast }, /* level 1 - in fact redirected towards zstd fast */ @@ -122,6 +147,7 @@ static const ZSTD_HC_parameters ZSTD_HC_defaultParameters[ZSTD_HC_MAX_CLEVEL+1] { 25, 24, 23, 5, 5, ZSTD_HC_btlazy2 }, /* level 18 */ { 25, 26, 23, 5, 5, ZSTD_HC_btlazy2 }, /* level 19 */ { 26, 27, 24, 6, 5, ZSTD_HC_btlazy2 }, /* level 20 */ +} }; diff --git a/programs/Makefile b/programs/Makefile index ea51bbf7..472222c4 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -30,7 +30,7 @@ # fullbench32: Same as fullbench, but forced to compile in 32-bits mode # ########################################################################## -VERSION?= 0.3.5 +VERSION?= 0.3.6 DESTDIR?= PREFIX ?= /usr/local diff --git a/programs/bench.c b/programs/bench.c index 0009e4ae..5d10f9fb 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -380,7 +380,7 @@ static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, i } -static U64 BMK_GetFileSize(char* infilename) +static U64 BMK_GetFileSize(const char* infilename) { int r; #if defined(_MSC_VER) diff --git a/programs/fileio.c b/programs/fileio.c index d168f8a4..4ef808fa 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -36,7 +36,7 @@ /**LEGACY_SUPPORT : * decompressor can decode older formats (starting from Zstd 0.1+) */ # define ZSTD_LEGACY_SUPPORT 1 -#endif // ZSTD_LEGACY_SUPPORT +#endif /* ************************************* @@ -58,11 +58,13 @@ /* ************************************* * Includes ***************************************/ -#include /* fprintf, fopen, fread, _fileno, stdin, stdout */ -#include /* malloc, free */ -#include /* strcmp, strlen */ -#include /* clock */ -#include /* errno */ +#include /* fprintf, fopen, fread, _fileno, stdin, stdout */ +#include /* malloc, free */ +#include /* strcmp, strlen */ +#include /* clock */ +#include /* errno */ +#include /* stat64 */ +#include /* stat64 */ #include "mem.h" #include "fileio.h" #include "zstd_static.h" @@ -91,6 +93,10 @@ # define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) #endif +#if !defined(S_ISREG) +# define S_ISREG(x) (((x) & S_IFMT) == S_IFREG) +#endif + /* ************************************* * Constants @@ -215,19 +221,35 @@ static void FIO_getFileHandles(FILE** pfinput, FILE** pfoutput, const char* inpu if ( *pfoutput==0) EXM_THROW(13, "Pb opening dst : %s", output_filename); } + +static U64 FIO_getFileSize(const char* infilename) +{ + int r; +#if defined(_MSC_VER) + struct _stat64 statbuf; + r = _stat64(infilename, &statbuf); +#else + struct stat statbuf; + r = stat(infilename, &statbuf); +#endif + if (r || !S_ISREG(statbuf.st_mode)) return 0; + return (U64)statbuf.st_size; +} + + typedef void* (*FIO_createC) (void); static void* local_ZSTD_createCCtx(void) { return (void*) ZSTD_createCCtx(); } static void* local_ZSTD_HC_createCCtx(void) { return (void*) ZSTD_HC_createCCtx(); } -typedef size_t (*FIO_initC) (void* ctx, void* dst, size_t maxDstSize, int cLevel); -static size_t local_ZSTD_compressBegin (void* ctx, void* dst, size_t maxDstSize, int cLevel) +typedef size_t (*FIO_initC) (void* ctx, void* dst, size_t maxDstSize, int cLevel, U64 srcSizeHint); +static size_t local_ZSTD_compressBegin (void* ctx, void* dst, size_t maxDstSize, int cLevel, U64 srcSizeHint) { - (void)cLevel; + (void)cLevel; (void)srcSizeHint; return ZSTD_compressBegin((ZSTD_CCtx*)ctx, dst, maxDstSize); } -static size_t local_ZSTD_HC_compressBegin (void* ctx, void* dst, size_t maxDstSize, int cLevel) +static size_t local_ZSTD_HC_compressBegin (void* ctx, void* dst, size_t maxDstSize, int cLevel, U64 srcSizeHint) { - return ZSTD_HC_compressBegin((ZSTD_HC_CCtx*)ctx, dst, maxDstSize, cLevel); + return ZSTD_HC_compressBegin((ZSTD_HC_CCtx*)ctx, dst, maxDstSize, cLevel, srcSizeHint); } typedef size_t (*FIO_continueC) (void* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); @@ -294,6 +316,7 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char* freeC = local_ZSTD_HC_freeCCtx; } FIO_getFileHandles(&finput, &foutput, input_filename, output_filename); + filesize = FIO_getFileSize(input_filename); /* Allocate Memory */ ctx = createC(); @@ -304,12 +327,13 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char* inEnd = inBuff + inBuffSize; /* Write Frame Header */ - cSize = initC(ctx, outBuff, outBuffSize, cLevel); + cSize = initC(ctx, outBuff, outBuffSize, cLevel, filesize); if (ZSTD_isError(cSize)) EXM_THROW(22, "Compression error : cannot create frame header"); sizeCheck = fwrite(outBuff, 1, cSize, foutput); if (sizeCheck!=cSize) EXM_THROW(23, "Write error : cannot write header into %s", output_filename); compressedfilesize += cSize; + filesize = 0; /* Main compression loop */ while (1) diff --git a/programs/fuzzer.c b/programs/fuzzer.c index ba402e71..7458bba8 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -355,7 +355,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit dSize = ZSTD_decompress(dstBuffer, sampleSize + dSupSize, cBuffer, cSize); CHECK(dSize != sampleSize, "ZSTD_decompress failed (%s) (srcSize : %u ; cSize : %u)", ZSTD_getErrorName(dSize), (U32)sampleSize, (U32)cSize); crcDest = XXH64(dstBuffer, sampleSize, 0); - CHECK(crcOrig != crcDest, "dstBuffer corrupted (pos %u / %u)", (U32)findDiff(srcBuffer+sampleStart, dstBuffer, sampleSize), (U32)sampleSize); + CHECK(crcOrig != crcDest, "decompression result corrupted (pos %u / %u)", (U32)findDiff(srcBuffer+sampleStart, dstBuffer, sampleSize), (U32)sampleSize); /* truncated src decompression test */ { diff --git a/programs/paramgrill.c b/programs/paramgrill.c index eb265e22..90cfdd77 100644 --- a/programs/paramgrill.c +++ b/programs/paramgrill.c @@ -122,7 +122,7 @@ static U32 g_rand = 1; static U32 g_singleRun = 0; static U32 g_target = 0; static U32 g_noSeed = 0; -static const ZSTD_HC_parameters* g_seedParams = ZSTD_HC_defaultParameters; +static const ZSTD_HC_parameters* g_seedParams = ZSTD_HC_defaultParameters[0]; static ZSTD_HC_parameters g_params = { 0, 0, 0, 0, 0, ZSTD_HC_greedy }; void BMK_SetNbIterations(int nbLoops) @@ -703,12 +703,13 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize) int i; const char* rfName = "grillResults.txt"; FILE* f; - const U32 srcLog = BMK_highbit((U32)( (g_blockSize ? g_blockSize : srcSize) -1))+1; + const size_t blockSize = g_blockSize ? g_blockSize : srcSize; + const U32 srcLog = BMK_highbit((U32)(blockSize-1))+1; if (g_singleRun) { BMK_result_t testResult; - ZSTD_HC_validateParams(&g_params, g_blockSize ? g_blockSize : srcSize); + ZSTD_HC_validateParams(&g_params, blockSize); BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, g_params); DISPLAY("\n"); return; @@ -725,12 +726,13 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize) { /* baseline config for level 1 */ BMK_result_t testResult; - params.windowLog = MIN(srcLog, 18); + params.windowLog = 18; params.hashLog = 14; params.contentLog = 1; params.searchLog = 1; params.searchLength = 7; params.strategy = ZSTD_HC_fast; + ZSTD_HC_validateParams(¶ms, blockSize); BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, params); g_cSpeedTarget[1] = (testResult.cSpeed * 15) >> 4; } @@ -741,7 +743,9 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize) /* populate initial solution */ { + const int tableID = (blockSize > 128 KB); const int maxSeeds = g_noSeed ? 1 : ZSTD_HC_MAX_CLEVEL; + g_seedParams = ZSTD_HC_defaultParameters[tableID]; for (i=1; i<=maxSeeds; i++) { const U32 btPlus = (params.strategy == ZSTD_HC_btlazy2); diff --git a/programs/zstdcli.c b/programs/zstdcli.c index c9946c4d..65b23550 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -70,7 +70,7 @@ **************************************/ #define COMPRESSOR_NAME "zstd command line interface" #ifndef ZSTD_VERSION -# define ZSTD_VERSION "v0.3.5" +# define ZSTD_VERSION "v0.3.6" #endif #define AUTHOR "Yann Collet" #define WELCOME_MESSAGE "*** %s %i-bits %s, by %s (%s) ***\n", COMPRESSOR_NAME, (int)(sizeof(void*)*8), ZSTD_VERSION, AUTHOR, __DATE__