From 5ddcd9d9ae438bc497c39a0c4cf095fcbcfde8ad Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Mon, 21 Nov 2016 16:37:56 +0100 Subject: [PATCH 1/5] bench.c: fixed MAX_CLEVEL --- programs/bench.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/programs/bench.c b/programs/bench.c index df68d0ad..32d3a5dc 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -483,12 +483,18 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility free(srcBuffer); } +#define ZSTD_MAX_CLEVEL 22 int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName, int cLevel, int cLevelLast) { double const compressibility = (double)g_compressibilityDefault / 100; + if (cLevel > ZSTD_MAX_CLEVEL) cLevel = ZSTD_MAX_CLEVEL; + if (cLevelLast > ZSTD_MAX_CLEVEL) cLevelLast = ZSTD_MAX_CLEVEL; + if (cLevelLast < cLevel) cLevelLast = cLevel; + if (cLevelLast > cLevel) DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast); + if (nbFiles == 0) BMK_syntheticTest(cLevel, cLevelLast, compressibility); else From ad3e94512c3f53850cbe200f222720a4edf7ba8f Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Mon, 21 Nov 2016 20:22:12 +0100 Subject: [PATCH 2/5] fixed warnings from static analyzer in zstd_opt.h --- lib/compress/zstd_opt.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/compress/zstd_opt.h b/lib/compress/zstd_opt.h index 90d511c7..53cd2326 100644 --- a/lib/compress/zstd_opt.h +++ b/lib/compress/zstd_opt.h @@ -416,7 +416,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, /* check repCode */ { U32 i, last_i = ZSTD_REP_CHECK + (ip==anchor); for (i=(ip == anchor); i 0) && (repCur < (S32)(ip-prefixStart)) && (MEM_readMINMATCH(ip, minMatch) == MEM_readMINMATCH(ip - repCur, minMatch))) { mlen = (U32)ZSTD_count(ip+minMatch, ip+minMatch-repCur, iend) + minMatch; @@ -501,7 +501,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, best_mlen = minMatch; { U32 i, last_i = ZSTD_REP_CHECK + (mlen != 1); for (i=(opt[cur].mlen != 1); i 0) && (repCur < (S32)(inr-prefixStart)) && (MEM_readMINMATCH(inr, minMatch) == MEM_readMINMATCH(inr - repCur, minMatch))) { mlen = (U32)ZSTD_count(inr+minMatch, inr+minMatch - repCur, iend) + minMatch; @@ -601,7 +601,7 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ offset--; } else { if (offset != 0) { - best_off = ((offset==ZSTD_REP_MOVE_OPT) && (litLength==0)) ? (rep[0] - 1) : (rep[offset]); + best_off = (offset==ZSTD_REP_MOVE_OPT) ? (rep[0] - 1) : (rep[offset]); if (offset != 1) rep[2] = rep[1]; rep[1] = rep[0]; rep[0] = best_off; @@ -671,7 +671,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, /* check repCode */ { U32 i, last_i = ZSTD_REP_CHECK + (ip==anchor); for (i = (ip==anchor); i Date: Wed, 23 Nov 2016 17:22:54 +0100 Subject: [PATCH 3/5] zstd_opt.h: improved price function --- lib/common/zstd_internal.h | 1 + lib/compress/zstd_opt.h | 47 ++++++++++++++++++++++++++------------ 2 files changed, 34 insertions(+), 14 deletions(-) diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index 266209e4..a5002fb1 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -222,6 +222,7 @@ typedef struct { U32 log2litSum; U32 log2offCodeSum; U32 factor; + U32 staticPrices; U32 cachedPrice; U32 cachedLitLength; const BYTE* cachedLiterals; diff --git a/lib/compress/zstd_opt.h b/lib/compress/zstd_opt.h index 53cd2326..ef17f796 100644 --- a/lib/compress/zstd_opt.h +++ b/lib/compress/zstd_opt.h @@ -15,8 +15,9 @@ #define ZSTD_OPT_H_91842398743 -#define ZSTD_FREQ_DIV 5 -#define ZSTD_MAX_PRICE (1<<30) +#define ZSTD_LITFREQ_ADD 2 +#define ZSTD_FREQ_DIV 4 +#define ZSTD_MAX_PRICE (1<<30) /*-************************************* * Price functions for optimal parser @@ -31,22 +32,32 @@ FORCE_INLINE void ZSTD_setLog2Prices(seqStore_t* ssPtr) } -MEM_STATIC void ZSTD_rescaleFreqs(seqStore_t* ssPtr) +MEM_STATIC void ZSTD_rescaleFreqs(seqStore_t* ssPtr, const BYTE* src, size_t srcSize) { unsigned u; ssPtr->cachedLiterals = NULL; ssPtr->cachedPrice = ssPtr->cachedLitLength = 0; + ssPtr->staticPrices = 0; if (ssPtr->litLengthSum == 0) { - ssPtr->litSum = (2<staticPrices = 1; + + for (u=0; u<=MaxLit; u++) + ssPtr->litFreq[u] = 0; + for (u=0; ulitFreq[src[u]]++; + + ssPtr->litSum = 0; ssPtr->litLengthSum = MaxLL+1; ssPtr->matchLengthSum = MaxML+1; ssPtr->offCodeSum = (MaxOff+1); - ssPtr->matchSum = (2<matchSum = (ZSTD_LITFREQ_ADD<litFreq[u] = 2; + for (u=0; u<=MaxLit; u++) { + ssPtr->litFreq[u] = 1 + (ssPtr->litFreq[u]>>ZSTD_FREQ_DIV); + ssPtr->litSum += ssPtr->litFreq[u]; + } for (u=0; u<=MaxLL; u++) ssPtr->litLengthFreq[u] = 1; for (u=0; u<=MaxML; u++) @@ -61,11 +72,11 @@ MEM_STATIC void ZSTD_rescaleFreqs(seqStore_t* ssPtr) ssPtr->litSum = 0; for (u=0; u<=MaxLit; u++) { - ssPtr->litFreq[u] = 1 + (ssPtr->litFreq[u]>>ZSTD_FREQ_DIV); + ssPtr->litFreq[u] = 1 + (ssPtr->litFreq[u]>>(ZSTD_FREQ_DIV+1)); ssPtr->litSum += ssPtr->litFreq[u]; } for (u=0; u<=MaxLL; u++) { - ssPtr->litLengthFreq[u] = 1 + (ssPtr->litLengthFreq[u]>>ZSTD_FREQ_DIV); + ssPtr->litLengthFreq[u] = 1 + (ssPtr->litLengthFreq[u]>>(ZSTD_FREQ_DIV+1)); ssPtr->litLengthSum += ssPtr->litLengthFreq[u]; } for (u=0; u<=MaxML; u++) { @@ -73,6 +84,7 @@ MEM_STATIC void ZSTD_rescaleFreqs(seqStore_t* ssPtr) ssPtr->matchLengthSum += ssPtr->matchLengthFreq[u]; ssPtr->matchSum += ssPtr->matchLengthFreq[u] * (u + 3); } + ssPtr->matchSum *= ZSTD_LITFREQ_ADD; for (u=0; u<=MaxOff; u++) { ssPtr->offCodeFreq[u] = 1 + (ssPtr->offCodeFreq[u]>>ZSTD_FREQ_DIV); ssPtr->offCodeSum += ssPtr->offCodeFreq[u]; @@ -87,6 +99,9 @@ FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* ssPtr, U32 litLength, const BY { U32 price, u; + if (ssPtr->staticPrices) + return ZSTD_highbit32((U32)litLength+1) + (litLength*6); + if (litLength == 0) return ssPtr->log2litLengthSum - ZSTD_highbit32(ssPtr->litLengthFreq[0]+1); @@ -124,9 +139,13 @@ FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* ssPtr, U32 litLength, const BY FORCE_INLINE U32 ZSTD_getPrice(seqStore_t* seqStorePtr, U32 litLength, const BYTE* literals, U32 offset, U32 matchLength, const int ultra) { /* offset */ + U32 price; BYTE const offCode = (BYTE)ZSTD_highbit32(offset+1); - U32 price = offCode + seqStorePtr->log2offCodeSum - ZSTD_highbit32(seqStorePtr->offCodeFreq[offCode]+1); + if (seqStorePtr->staticPrices) + return ZSTD_getLiteralPrice(seqStorePtr, litLength, literals) + ZSTD_highbit32((U32)matchLength+1) + 16 + offCode; + + price = offCode + seqStorePtr->log2offCodeSum - ZSTD_highbit32(seqStorePtr->offCodeFreq[offCode]+1); if (!ultra && offCode >= 20) price += (offCode-19)*2; /* match Length */ @@ -144,9 +163,9 @@ MEM_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const B U32 u; /* literals */ - seqStorePtr->litSum += litLength; + seqStorePtr->litSum += litLength*ZSTD_LITFREQ_ADD; for (u=0; u < litLength; u++) - seqStorePtr->litFreq[literals[u]]++; + seqStorePtr->litFreq[literals[u]] += ZSTD_LITFREQ_ADD; /* literal Length */ { const BYTE LL_deltaCode = 19; @@ -401,7 +420,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, /* init */ ctx->nextToUpdate3 = ctx->nextToUpdate; - ZSTD_rescaleFreqs(seqStorePtr); + ZSTD_rescaleFreqs(seqStorePtr, src, srcSize); ip += (ip==prefixStart); { U32 i; for (i=0; irep[i]; } @@ -656,7 +675,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, { U32 i; for (i=0; irep[i]; } ctx->nextToUpdate3 = ctx->nextToUpdate; - ZSTD_rescaleFreqs(seqStorePtr); + ZSTD_rescaleFreqs(seqStorePtr, src, srcSize); ip += (ip==prefixStart); /* Match Loop */ From fc4193bda5e8c0c46de9da304215ec219e49419c Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 23 Nov 2016 18:17:18 +0100 Subject: [PATCH 4/5] fixed g++ warnings --- lib/compress/zstd_opt.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/compress/zstd_opt.h b/lib/compress/zstd_opt.h index ef17f796..f071c4f3 100644 --- a/lib/compress/zstd_opt.h +++ b/lib/compress/zstd_opt.h @@ -420,7 +420,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, /* init */ ctx->nextToUpdate3 = ctx->nextToUpdate; - ZSTD_rescaleFreqs(seqStorePtr, src, srcSize); + ZSTD_rescaleFreqs(seqStorePtr, (const BYTE*)src, srcSize); ip += (ip==prefixStart); { U32 i; for (i=0; irep[i]; } @@ -675,7 +675,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, { U32 i; for (i=0; irep[i]; } ctx->nextToUpdate3 = ctx->nextToUpdate; - ZSTD_rescaleFreqs(seqStorePtr, src, srcSize); + ZSTD_rescaleFreqs(seqStorePtr, (const BYTE*)src, srcSize); ip += (ip==prefixStart); /* Match Loop */ From fd0ac93024b074d50f7b8e9d46e1a22720654719 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 23 Nov 2016 21:45:29 +0100 Subject: [PATCH 5/5] bench.c: use ZSTD_maxCLevel() --- programs/bench.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 32d3a5dc..5f31bb4b 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -483,15 +483,14 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility free(srcBuffer); } -#define ZSTD_MAX_CLEVEL 22 int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName, int cLevel, int cLevelLast) { double const compressibility = (double)g_compressibilityDefault / 100; - if (cLevel > ZSTD_MAX_CLEVEL) cLevel = ZSTD_MAX_CLEVEL; - if (cLevelLast > ZSTD_MAX_CLEVEL) cLevelLast = ZSTD_MAX_CLEVEL; + if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel(); + if (cLevelLast > ZSTD_maxCLevel()) cLevelLast = ZSTD_maxCLevel(); if (cLevelLast < cLevel) cLevelLast = cLevel; if (cLevelLast > cLevel) DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast);